{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b5e30b7c-e5b8-4119-8ac0-486a18e8e9c4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-06T11:49:52.914025Z",
     "iopub.status.busy": "2022-07-06T11:49:52.913795Z",
     "iopub.status.idle": "2022-07-06T11:50:39.109091Z",
     "shell.execute_reply": "2022-07-06T11:50:39.108269Z",
     "shell.execute_reply.started": "2022-07-06T11:49:52.914002Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5c8cf960c3f46ef87966968e1b40962",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Spark application\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n<tbody><tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>User</th><th>Current session?</th></tr><tr><td>0</td><td>application_1657076023788_0001</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://ip-172-31-28-185.ec2.internal:20888/proxy/application_1657076023788_0001/\" class=\"emr-proxy-link j-2NSZQRHJKMESS application_1657076023788_0001\" emr-resource=\"j-2NSZQRHJKMESS\n\" application-id=\"application_1657076023788_0001\">Link</a></td><td><a target=\"_blank\" href=\"http://ip-172-31-23-115.ec2.internal:8042/node/containerlogs/container_1657076023788_0001_01_000001/livy\">Link</a></td><td>None</td><td>✔</td></tr></tbody></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparkSession available as 'spark'.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pyspark.sql.session.SparkSession object at 0x7fed0d579e10>"
     ]
    }
   ],
   "source": [
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "40b4f7d7-0bbc-4f0e-b7e1-6b6bb0c705dc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-06T12:44:22.560112Z",
     "iopub.status.busy": "2022-07-06T12:44:22.559854Z",
     "iopub.status.idle": "2022-07-06T12:44:39.932770Z",
     "shell.execute_reply": "2022-07-06T12:44:39.932180Z",
     "shell.execute_reply.started": "2022-07-06T12:44:22.560088Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aba3c81a67bc43f6b3f974003cc35746",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------+\n",
      "|    product_category|   count|\n",
      "+--------------------+--------+\n",
      "|               Music| 6177781|\n",
      "|                Home| 6228567|\n",
      "|Health_&_Personal...| 5332883|\n",
      "|               Books|20726160|\n",
      "|           Video_DVD| 7135819|\n",
      "|                  PC| 7004337|\n",
      "|              Beauty| 5115721|\n",
      "|              Sports| 4860054|\n",
      "|           Gift_Card|  149086|\n",
      "|            Wireless| 9038249|\n",
      "|Personal_Care_App...|   86686|\n",
      "|    Major_Appliances|   96901|\n",
      "|         Electronics| 3120938|\n",
      "|               Shoes| 4379475|\n",
      "|               Video|  437409|\n",
      "|          Automotive| 3516476|\n",
      "|             Watches|  978042|\n",
      "|     Office_Products| 2646491|\n",
      "|        Pet_Products| 2643670|\n",
      "|Digital_Music_Pur...| 1852184|\n",
      "+--------------------+--------+\n",
      "only showing top 20 rows"
     ]
    }
   ],
   "source": [
    "df = spark.read.parquet(\"s3://dsoaws/parquet\").groupBy(\"product_category\").count().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7b3020b-f23c-438c-9711-e80469f31e15",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PySpark",
   "language": "python",
   "name": "pysparkkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "python",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "pyspark",
   "pygments_lexer": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
